
/** 
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package com.tompai.datagather.crawldb;


import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.tompai.datagather.conf.Configuration;
import com.tompai.datagather.conf.DefaultConfigured;
import com.tompai.datagather.model.CrawlDatum;

/**
* @author: tompai
* @createTime: 2018年4月28日 下午7:42:41
* @history:
* @version: v1.0
*/
public abstract class Generator extends DefaultConfigured{

    public static final Logger LOG = LoggerFactory.getLogger(Generator.class);


    protected GeneratorFilter filter = null;
    protected int topN = 0;

    protected int totalGenerate;
    protected int maxExecuteCount;

    public Generator() {
        this.totalGenerate = 0;
        this.maxExecuteCount = getConf().getOrDefault(Configuration.KEY_MAX_EXECUTE_COUNT, Integer.MAX_VALUE);
    }

    /**
     * return null if there is no CrawlDatum to generate
     * @return
     */
    public CrawlDatum next(){
        if(topN > 0 && totalGenerate >= topN){
            return null;
        }
        CrawlDatum datum;
        while (true) {
            try {
                datum = nextWithoutFilter();
                if (datum == null) {
                    return datum;
                }
                if(filter == null || (datum = filter.filter(datum))!=null){
                    if (datum.getExecuteCount() > maxExecuteCount) {
                        continue;
                    }
                    totalGenerate += 1;
                    return datum;
                }

            } catch (Exception e) {
                LOG.info("Exception when generating", e);
                return null;
            }

        }
    }

    public abstract CrawlDatum nextWithoutFilter() throws Exception;


    public int getTopN() {
        return topN;
    }

    public void setTopN(int topN) {
        this.topN = topN;
    }


    public int getMaxExecuteCount() {
        return maxExecuteCount;
    }

    public void setMaxExecuteCount(int maxExecuteCount) {
        this.maxExecuteCount = maxExecuteCount;
    }

    public int getTotalGenerate(){
        return totalGenerate;
    }

    public abstract void close() throws Exception;

    public GeneratorFilter getFilter() {
        return filter;
    }

    public void setFilter(GeneratorFilter filter) {
        this.filter = filter;
    }
}
